# -*- coding: utf-8 -*-
import scrapy
from ..items import Cmdy5Item
from scrapy.loader import ItemLoader
from bs4 import BeautifulSoup
from user_agent import generate_user_agent

class Cmdy5Spider(scrapy.Spider):
    name = 'cmdy5'
    allowed_domains = ['cmdy5.com']
    base = 'https://www.cmdy5.com'
    start_urls = ['https://www.cmdy5.com/dianying.html']

    def parse(self, response):
        html = response.body_as_unicode()
        soup = BeautifulSoup(html, 'html.parser')

        print(soup.text)

        moive_list = soup.find('div', class_='index-area').find_all('li')
        for moive in moive_list:
        	mitem = Cmdy5Item()
        	mitem['name'] = 'cmdy5'
        	mitem['title'] = moive.a.get('title')
        	mitem['img'] = moive.find("img").get("data-original")
        	mitem['url'] = moive.a.get('href')

        	yield mitem

        # 下一页
        next_page = soup.find('a', class_='pagelink_a').get('href')
        if next_page:
        	next_url = self.base + next_page
        	headers = {
        		'User-Agent': generate_user_agent(device_type=['desktop'])
        	}
        	yield scrapy.Request(next_url, headers=headers, callback=self.parse)

